Concrete Stregth Prediction¶
In [97]:
import warnings
warnings.filterwarnings('ignore')
import random
import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
%matplotlib inline
plt.style.use('ggplot')
import seaborn as sns
from IPython.core.display import display, HTML
display(HTML("<style>.container { width:95% !important; }</style>"))
# Import Linear Regression machine learning library
from sklearn.linear_model import LinearRegression, Ridge, Lasso
from sklearn.model_selection import train_test_split
from sklearn.model_selection import KFold
from sklearn.model_selection import cross_val_score
from sklearn.model_selection import GridSearchCV
from sklearn.model_selection import RandomizedSearchCV
from sklearn.tree import DecisionTreeRegressor
from sklearn.ensemble import RandomForestRegressor
Cement (component 1) -- quantitative -- kg in a m3 mixture -- Input Variable
Blast Furnace Slag (component 2) -- quantitative -- kg in a m3 mixture -- Input Variable
Fly Ash (component 3) -- quantitative -- kg in a m3 mixture -- Input Variable
Water (component 4) -- quantitative -- kg in a m3 mixture -- Input Variable
Superplasticizer (component 5) -- quantitative -- kg in a m3 mixture -- Input Variable
Coarse Aggregate (component 6) -- quantitative -- kg in a m3 mixture -- Input Variable
Fine Aggregate (component 7) -- quantitative -- kg in a m3 mixture -- Input Variable
Age -- quantitative -- Day (1~365) -- Input Variable
Concrete compressive strength -- quantitative -- MPa -- Output Variable
In [98]:
# importing data
df = pd.read_csv("concrete.csv")
df.head() # used to see top 5 rows of the dataset
Out[98]:
1. Univariate analysis (10 )¶
In [99]:
#five point analysis
df.describe().transpose()
Out[99]:
In [100]:
#checking for datatypes
df.info()
In [101]:
#no of rows and columns
df.shape
Out[101]:
In [102]:
#checking for null values
df.isnull().values.any()
Out[102]:
In [103]:
#checking for any skewed values
df.skew()
Out[103]:
In [104]:
#visualizing age skewness using a box plot
sns.boxplot(df['age'])
Out[104]:
In [105]:
sns.distplot(df['age'])
# Age value less than 365 is allowed since the age value in this can range (1~365)
Out[105]:
In [106]:
sns.distplot(df['fineagg'])
Out[106]:
In [107]:
sns.distplot(df['coarseagg'])
Out[107]:
In [108]:
sns.distplot(df['superplastic'])
Out[108]:
In [109]:
sns.distplot(df['water'])
Out[109]:
In [110]:
sns.distplot(df['ash'])
Out[110]:
In [111]:
sns.distplot(df['slag'])
Out[111]:
In [112]:
sns.distplot(df['cement'])
Out[112]:
In [113]:
sns.distplot(df['strength'], kde= True)
Out[113]:
2.Bivariate Analysis (10 )¶
In [114]:
sns.pairplot(df)
#From the pariplot as cement increases strength also increase
Out[114]:
In [115]:
# visualise area-price relationship
sns.regplot(x="cement", y="strength", data=df, fit_reg=False)
Out[115]:
In [116]:
sns.regplot(x="slag", y="strength", data=df, fit_reg=False)
Out[116]:
In [117]:
sns.regplot(x="ash", y="strength", data=df, fit_reg=False)
Out[117]:
In [118]:
sns.regplot(x="water", y="strength", data=df, fit_reg=False)
Out[118]:
In [119]:
sns.regplot(x="superplastic", y="strength", data=df, fit_reg=False)
Out[119]:
In [120]:
sns.regplot(x="coarseagg", y="strength", data=df, fit_reg=False)
Out[120]:
In [121]:
sns.regplot(x="fineagg", y="strength", data=df, fit_reg=False)
Out[121]:
In [122]:
sns.regplot(x="age", y="strength", data=df, fit_reg=False)
Out[122]:
3. Feature engineering techniques (10 )¶
In [123]:
#Checking for highly corelaed variables
df.corr()
#no highly correlated independent variables so planning to keep all varibales as it is.
Out[123]:
In [124]:
plt.figure(figsize=(10,8))
sns.heatmap(df.corr(),
annot=True,
linewidths=.5,
center=0,
cbar=False)
plt.show()
In [125]:
# Copy all the predictor variables into X dataframe. Since 'strength' is dependent variable drop it
X = df.drop('strength', axis=1)
# Copy the 'strength' column alone into the y dataframe. This is the dependent variable
y = df[['strength']]
X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.30, random_state=1)
In [126]:
#Using polynomial Features to create more independent variables
from sklearn.preprocessing import PolynomialFeatures
poly = PolynomialFeatures(degree = 2, interaction_only=True)
X_poly = poly.fit_transform(X)
X_train, X_test, y_train, y_test = train_test_split(X_poly, y, test_size=0.30, random_state=1)
4 .Creating the model and tuning it (30 )¶
In [127]:
#Linear Regression
regression_model = LinearRegression()
regression_model.fit(X_train, y_train)
regression_model.score(X_test, y_test)
Out[127]:
In [128]:
#Ridge
ridge = Ridge(alpha=.3)
ridge.fit(X_train,y_train)
print ("Ridge model:", (ridge.coef_))
In [129]:
#Lasso ; Just seeing if we can drop any independent variable.
lasso = Lasso(alpha=0.1)
lasso.fit(X_train,y_train)
print ("Lasso model:", (lasso.coef_))
In [130]:
print(ridge.score(X_train, y_train))
print(ridge.score(X_test, y_test))
In [131]:
print(lasso.score(X_train, y_train))
print(lasso.score(X_test, y_test))
#The test scores are comparable between simple and quadratic models. Scores are coming around 60%. We have to try other algorithms
In [132]:
dtRegressor = DecisionTreeRegressor(random_state=0)
dtRegressor.fit(X_train,y_train)
print(dtRegressor.score(X_train, y_train))
print(dtRegressor.score(X_test, y_test))
#Decsion tree regressor is giving the required scores before applying cross validation
In [133]:
randomForestRegressor = RandomForestRegressor(max_depth=2, random_state=0)
randomForestRegressor.fit(X_train,y_train)
print(randomForestRegressor.score(X_train, y_train))
print(randomForestRegressor.score(X_test, y_test))
In [134]:
from sklearn.ensemble import AdaBoostRegressor
abRegressor = AdaBoostRegressor(random_state=0, n_estimators=100)
abRegressor.fit(X_train,y_train)
print(abRegressor.score(X_train, y_train))
print(abRegressor.score(X_test, y_test))
In [135]:
from sklearn.ensemble import GradientBoostingRegressor
gradientRegressor = GradientBoostingRegressor(random_state=0)
gradientRegressor.fit(X_train,y_train)
print(gradientRegressor.score(X_train, y_train))
print(gradientRegressor.score(X_test, y_test))
So far we had got good results in DecisionTreeRegressor and Gradient Boosting Regressor ~82% and ~96% on training.¶
We will take DecisionTreeRegressor and Gradient Boosting Regressor for hyperparameter tuning and cross validation.¶
We will find out the best combination of hyperparameters and then do the KFold crossvalidation on those before concluding on the result¶
DecisionTree Regressor¶
In [136]:
dtRegressor.get_params().keys()
Out[136]:
In [137]:
#We will try to use RandomSearchCV
samples = 10 # number of random samples
depths = np.arange(1, 20)
num_leafs = [1, 5, 10, 20]
parameters = [{'max_depth':depths,
'min_samples_leaf':num_leafs}]
dtRegressorRSCV = RandomizedSearchCV(dtRegressor, parameters, cv=samples)
dtRegressorRSCV.fit(X, y)
print(dtRegressorRSCV.best_params_)
In [138]:
num_folds = 3
seed = 7
kfold = KFold(n_splits=num_folds, random_state=seed)
results = cross_val_score(dtRegressorRSCV, X, y, cv=kfold, scoring='r2')
print(results)
print("Accuracy: %.3f%% (%.3f%%)" % (results.mean()*100.0, results.std()*100.0))
In [139]:
#We will try to use GridSearchCV now
samples = 10 # number of random samples
depths = np.arange(1, 20)
num_leafs = [1, 5, 10, 20]
parameters = { 'min_samples_leaf' : num_leafs,
'max_depth' : depths
}
dtRegressorGSCV = GridSearchCV(dtRegressor, param_grid = parameters, scoring='r2', cv=samples)
dtRegressorGSCV.fit(X, y)
print(dtRegressorGSCV.best_params_)
In [140]:
num_folds = 3
seed = 7
kfold = KFold(n_splits=num_folds, random_state=seed)
results = cross_val_score(dtRegressorGSCV, X, y, cv=kfold, scoring='r2')
print(results)
print("Accuracy: %.3f%% (%.3f%%)" % (results.mean()*100.0, results.std()*100.0))
Gradient Regressor¶
In [141]:
gradientRegressor.get_params().keys()
Out[141]:
In [142]:
#We will try to use RandomSearchCV
samples = 10 # number of random samples
depths = np.arange(1, 20)
num_leafs = [ 20, 30 ]
parameters = [{'max_depth':depths,
'min_samples_leaf':num_leafs}]
gradientRegressorRSCV = RandomizedSearchCV(gradientRegressor, parameters, cv=samples)
gradientRegressorRSCV.fit(X, y)
print(gradientRegressorRSCV.best_params_)
In [143]:
num_folds = 3
seed = 7
kfold = KFold(n_splits=num_folds, random_state=seed)
results = cross_val_score(gradientRegressorRSCV, X, y, cv=kfold, scoring='r2')
print(results)
print("Accuracy: %.3f%% (%.3f%%)" % (results.mean()*100.0, results.std()*100.0))
In [144]:
#We will try to use GridSearchCV
samples = 10 # number of random samples
depths = np.arange(1, 20)
num_leafs = [ 20 , 30]
parameters = { 'max_depth' : depths,
'min_samples_leaf': num_leafs
}
gradientRegressorGSCV = GridSearchCV(gradientRegressor, param_grid = parameters, scoring='r2', cv=samples)
gradientRegressorGSCV.fit(X, y)
print(gradientRegressorGSCV.best_params_)
In [145]:
num_folds = 3
seed = 7
kfold = KFold(n_splits=num_folds, random_state=seed)
results = cross_val_score(gradientRegressorGSCV, X, y, cv=kfold, scoring='r2')
print(results)
print("Accuracy: %.3f%% (%.3f%%)" % (results.mean()*100.0, results.std()*100.0))